{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from __future__ import absolute_import\n",
    "from __future__ import division\n",
    "from __future__ import print_function\n",
    "\n",
    "import argparse\n",
    "import sys\n",
    "\n",
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "\n",
    "import tensorflow as tf\n",
    "\n",
    "FLAGS = None\n",
    "if 'session' in locals() and session is not None:\n",
    "    print('Close interactive session')\n",
    "    session.close()\n",
    "gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.33)\n",
    "sess = tf.Session(config=tf.ConfigProto(gpu_options=gpu_options))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting /tmp/tensorflow/mnist/input_data\\train-images-idx3-ubyte.gz\n",
      "Extracting /tmp/tensorflow/mnist/input_data\\train-labels-idx1-ubyte.gz\n",
      "Extracting /tmp/tensorflow/mnist/input_data\\t10k-images-idx3-ubyte.gz\n",
      "Extracting /tmp/tensorflow/mnist/input_data\\t10k-labels-idx1-ubyte.gz\n"
     ]
    }
   ],
   "source": [
    "# Import data\n",
    "data_dir = '/tmp/tensorflow/mnist/input_data'\n",
    "mnist = input_data.read_data_sets(data_dir, one_hot=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 设置训练参数\n",
    "in_units = 784 # 28*28\n",
    "out_units = 10\n",
    "\n",
    "h1_layers = 300 #500=>400=>300\n",
    "h2_layers = 100 #300=>200=>100\n",
    "\n",
    "learning_rate = 0.8 #0.1=>0.5=>0.9=>0.8\n",
    "learning_rate1 = 0.001 #0.1 => 0.01 => 0.005 => 0.001\n",
    "regularizaton_factor = 0.001 # 0.05 => 0.01 => 0.001\n",
    "\n",
    "train_steps = 16000\n",
    "batch_size = 400"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.223164\n",
      "0.966455\n",
      "0.983073\n",
      "0.988346\n",
      "0.991855\n",
      "0.995182\n",
      "0.996728\n",
      "0.997382\n",
      "0.997855\n",
      "0.998182\n",
      "0.998546\n",
      "0.998837\n",
      "0.999018\n",
      "0.999073\n",
      "0.999182\n",
      "0.999237\n",
      "0.998128\n",
      "0.999218\n",
      "0.999346\n",
      "0.999382\n",
      "0.999418\n",
      "0.999491\n",
      "0.999509\n",
      "0.999546\n",
      "0.994928\n",
      "0.999546\n",
      "0.999582\n",
      "0.9996\n",
      "0.9996\n",
      "0.999618\n",
      "0.999673\n",
      "0.999673\n",
      "0.999691\n",
      "0.999691\n",
      "0.997273\n",
      "0.9996\n",
      "0.999691\n",
      "0.999727\n",
      "0.999727\n",
      "0.999727\n",
      "0.999746\n",
      "0.999746\n",
      "0.999764\n",
      "0.999764\n",
      "0.999764\n",
      "0.999764\n",
      "0.999782\n",
      "0.999782\n",
      "0.999782\n",
      "0.999782\n",
      "0.999273\n",
      "0.999746\n",
      "0.999818\n",
      "0.999818\n"
     ]
    }
   ],
   "source": [
    "def add_layer(inputs, in_size, out_size, activation_function=None):\n",
    "        # 权重参数初始化方式 正态分布 => 均匀分布 => 截断正态分布\n",
    "#         Weights =tf.Variable(tf.random_normal([in_size, out_size]), name='W')\n",
    "        \n",
    "#         Weights =tf.Variable(tf.random_uniform([in_size, out_size], -1.0, 1.0), name='W')\n",
    "        \n",
    "        Weights =tf.Variable(tf.truncated_normal([in_size, out_size],stddev=0.1), name='W')\n",
    "        \n",
    "        biases = tf.Variable(tf.zeros([1,out_size]) + 0.1, name='b')\n",
    "        \n",
    "        Wx_plus_b = tf.add(tf.matmul(inputs, Weights), biases)\n",
    "        \n",
    "        if activation_function is None:\n",
    "            outputs = Wx_plus_b\n",
    "        else:\n",
    "            outputs = activation_function(Wx_plus_b, )\n",
    "        return outputs , Weights\n",
    "\n",
    "# 网络结构 1隐层 => 2隐层\n",
    "# 输入\n",
    "xs = tf.placeholder(tf.float32, [None,in_units])\n",
    "ys = tf.placeholder(tf.float32, [None,out_units])\n",
    "# 隐层1 relu => relu6\n",
    "l1, weights1 = add_layer(xs, in_units, h1_layers,  activation_function = tf.nn.relu6)\n",
    "# 隐层2\n",
    "l2, weights2 = add_layer(l1, h1_layers, h2_layers,  activation_function = tf.nn.relu6)\n",
    "# 输出层\n",
    "prediction,weights3 = add_layer(l2, h2_layers, out_units,  activation_function= tf.nn.softmax)\n",
    "# 正则 l1 => l2\n",
    "regularizer = tf.contrib.layers.l2_regularizer(regularizaton_factor)\n",
    "regularizaton =  regularizer(weights3)\n",
    "# 损失\n",
    "loss = tf.reduce_mean(tf.reduce_sum(tf.square(ys - prediction), reduction_indices=[1])) + regularizaton\n",
    "# 损失优化过程中，GradientDescentOptimizer的学习率在0.8时比较合适，而使用 AdamOptimizer 只需要0.05的学习率就能达到不错的精确率，后者在优化过程中，收敛的速度更快\n",
    "# train_step =tf.train.GradientDescentOptimizer(learning_rate).minimize(loss)   \n",
    "train_step =tf.train.AdamOptimizer(learning_rate1).minimize(loss)\n",
    "\n",
    "\n",
    "correct_prediction = tf.equal(tf.argmax(prediction, 1), tf.argmax(ys, 1))  \n",
    "accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "\n",
    "sess = tf.Session()\n",
    "sess.run(tf.global_variables_initializer())\n",
    "\n",
    "for i in range(train_steps):\n",
    "    batch_xs, batch_ys = mnist.train.next_batch(batch_size)\n",
    "    sess.run(train_step, feed_dict = {xs:batch_xs, ys: batch_ys})\n",
    "    if i % 300 == 0:\n",
    "         print(sess.run(accuracy,feed_dict = {xs: mnist.train.images, ys: mnist.train.labels}))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9846\n"
     ]
    }
   ],
   "source": [
    "print(sess.run(accuracy, feed_dict={xs: mnist.test.images,\n",
    "                                      ys: mnist.test.labels}))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 调参过程发现的疑问：\n",
    "```\n",
    " 1、权重参数正则化，由于要对全网各层的权重参数都施加正则化惩罚，而我采用 regularizer(weights3) 只对输出层施加，\n",
    " 这种方式肯定是不合理的，对于是否可以进行更好的正则方式，请助教能否解答一二？\n",
    " 2、激活函数:在tf高版本 中使用用 swish 训练后发现性能与 relu 差不多,查资料说 swish 在深层模型中效果更佳,而我的网络很浅,应该是没有太大差异的 \n",
    "```"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
